in order to avoid leaking memory, remove all symbolization

Andrew Cantino 11 anni fa
parent
commit
77882908c4

+ 1 - 1
app/concerns/email_concern.rb

@@ -1,7 +1,7 @@
1 1
 module EmailConcern
2 2
   extend ActiveSupport::Concern
3 3
 
4
-  MAIN_KEYS = %w[title message text main value].map(&:to_sym)
4
+  MAIN_KEYS = %w[title message text main value]
5 5
 
6 6
   included do
7 7
     self.validate :validate_email_options

+ 3 - 3
app/models/agent.rb

@@ -1,14 +1,14 @@
1
-require 'serialize_and_symbolize'
1
+require 'serialize_and_normalize'
2 2
 require 'assignable_types'
3 3
 require 'markdown_class_attributes'
4 4
 require 'utils'
5 5
 
6 6
 class Agent < ActiveRecord::Base
7
-  include SerializeAndSymbolize
7
+  include SerializeAndNormalize
8 8
   include AssignableTypes
9 9
   include MarkdownClassAttributes
10 10
 
11
-  serialize_and_symbolize :options, :memory
11
+  serialize_and_normalize :options, :memory
12 12
   markdown_class_attributes :description, :event_description
13 13
 
14 14
   load_types_in "Agents"

+ 1 - 1
app/models/agents/adioso_agent.rb

@@ -44,7 +44,7 @@ module Agents
44 44
     end
45 45
 
46 46
     def validate_options
47
-			unless %w[start_date end_date from to username password expected_update_period_in_days].all? { |field| options[field.to_sym].present? }
47
+			unless %w[start_date end_date from to username password expected_update_period_in_days].all? { |field| options[field].present? }
48 48
 				errors.add(:base, "All fields are required")
49 49
 			end
50 50
 		end

+ 14 - 14
app/models/agents/human_task_agent.rb

@@ -214,12 +214,12 @@ module Agents
214 214
           end
215 215
 
216 216
           event = create_event :payload => payload
217
-          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym])
217
+          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id])
218 218
 
219 219
           assignments.each(&:approve!)
220 220
           hit.dispose!
221 221
 
222
-          memory[:hits].delete(hit_id.to_sym)
222
+          memory[:hits].delete(hit_id)
223 223
         end
224 224
       end
225 225
     end
@@ -268,34 +268,34 @@ module Agents
268 268
         @questions.each.with_index do |question, index|
269 269
           Question do
270 270
             QuestionIdentifier do
271
-              text question[:key] || "question_#{index}"
271
+              text question['key'] || "question_#{index}"
272 272
             end
273 273
             DisplayName do
274
-              text question[:name] || "Question ##{index}"
274
+              text question['name'] || "Question ##{index}"
275 275
             end
276 276
             IsRequired do
277
-              text question[:required] || 'true'
277
+              text question['required'] || 'true'
278 278
             end
279 279
             QuestionContent do
280 280
               Text do
281
-                text question[:question]
281
+                text question['question']
282 282
               end
283 283
             end
284 284
             AnswerSpecification do
285
-              if question[:type] == "selection"
285
+              if question['type'] == "selection"
286 286
 
287 287
                 SelectionAnswer do
288 288
                   StyleSuggestion do
289 289
                     text 'radiobutton'
290 290
                   end
291 291
                   Selections do
292
-                    question[:selections].each do |selection|
292
+                    question['selections'].each do |selection|
293 293
                       Selection do
294 294
                         SelectionIdentifier do
295
-                          text selection[:key]
295
+                          text selection['key']
296 296
                         end
297 297
                         Text do
298
-                          text selection[:text]
298
+                          text selection['text']
299 299
                         end
300 300
                       end
301 301
                     end
@@ -305,18 +305,18 @@ module Agents
305 305
               else
306 306
 
307 307
                 FreeTextAnswer do
308
-                  if question[:min_length].present? || question[:max_length].present?
308
+                  if question['min_length'].present? || question['max_length'].present?
309 309
                     Constraints do
310 310
                       lengths = {}
311
-                      lengths[:minLength] = question[:min_length].to_s if question[:min_length].present?
312
-                      lengths[:maxLength] = question[:max_length].to_s if question[:max_length].present?
311
+                      lengths['minLength'] = question['min_length'].to_s if question['min_length'].present?
312
+                      lengths['maxLength'] = question['max_length'].to_s if question['max_length'].present?
313 313
                       Length lengths
314 314
                     end
315 315
                   end
316 316
 
317 317
                   if question[:default].present?
318 318
                     DefaultText do
319
-                      text question[:default]
319
+                      text question['default']
320 320
                     end
321 321
                   end
322 322
                 end

+ 1 - 1
app/models/agents/peak_detector_agent.rb

@@ -114,7 +114,7 @@ module Agents
114 114
     end
115 115
 
116 116
     def group_for(event)
117
-      ((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group').to_sym
117
+      ((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group')
118 118
     end
119 119
 
120 120
     def remember(group, event)

+ 3 - 3
app/models/agents/twilio_agent.rb

@@ -79,9 +79,9 @@ module Agents
79 79
     end
80 80
 
81 81
     def receive_webhook(params)
82
-      if memory[:pending_calls].has_key? params[:secret].to_sym
83
-        response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret].to_sym], :voice => 'woman'}
84
-        memory[:pending_calls].delete params[:secret].to_sym
82
+      if memory[:pending_calls].has_key? params[:secret]
83
+        response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret]], :voice => 'woman'}
84
+        memory[:pending_calls].delete params[:secret]
85 85
         [response.text, 200]
86 86
       end
87 87
     end

+ 5 - 5
app/models/agents/twitter_stream_agent.rb

@@ -85,12 +85,12 @@ module Agents
85 85
           # Avoid memory pollution by reloading the Agent.
86 86
           agent = Agent.find(id)
87 87
           agent.memory[:filter_counts] ||= {}
88
-          agent.memory[:filter_counts][filter.to_sym] ||= 0
89
-          agent.memory[:filter_counts][filter.to_sym] += 1
88
+          agent.memory[:filter_counts][filter] ||= 0
89
+          agent.memory[:filter_counts][filter] += 1
90 90
           remove_unused_keys!(agent, :filter_counts)
91 91
           agent.save!
92 92
         else
93
-          create_event :payload => status.merge(:filter => filter.to_s)
93
+          create_event :payload => status.merge(:filter => filter)
94 94
         end
95 95
       end
96 96
     end
@@ -98,7 +98,7 @@ module Agents
98 98
     def check
99 99
       if options[:generate] == "counts" && memory[:filter_counts] && memory[:filter_counts].length > 0
100 100
         memory[:filter_counts].each do |filter, count|
101
-          create_event :payload => { :filter => filter.to_s, :count => count, :time => Time.now.to_i }
101
+          create_event :payload => { :filter => filter, :count => count, :time => Time.now.to_i }
102 102
         end
103 103
       end
104 104
       memory[:filter_counts] = {}
@@ -120,7 +120,7 @@ module Agents
120 120
 
121 121
     def remove_unused_keys!(agent, base)
122 122
       if agent.memory[base]
123
-        (agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_sym : f.to_sym }).each do |removed_key|
123
+        (agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key|
124 124
           agent.memory[base].delete(removed_key)
125 125
         end
126 126
       end

+ 5 - 7
app/models/event.rb

@@ -1,23 +1,21 @@
1
+require 'serialize_and_normalize'
2
+
1 3
 class Event < ActiveRecord::Base
4
+  include SerializeAndNormalize
5
+
2 6
   attr_accessible :lat, :lng, :payload, :user_id, :user, :expires_at
3 7
 
4 8
   acts_as_mappable
5 9
 
6
-  serialize :payload
10
+  serialize_and_normalize :payload
7 11
 
8 12
   belongs_to :user
9 13
   belongs_to :agent, :counter_cache => true
10 14
 
11
-  before_save :symbolize_payload
12
-
13 15
   scope :recent, lambda { |timespan = 12.hours.ago|
14 16
     where("events.created_at > ?", timespan)
15 17
   }
16 18
 
17
-  def symbolize_payload
18
-    self.payload = payload.recursively_symbolize_keys if payload.is_a?(Hash)
19
-  end
20
-
21 19
   def reemit!
22 20
     agent.create_event :payload => payload, :lat => lat, :lng => lng
23 21
   end

+ 0 - 7
config/initializers/recursively_symbolize_keys.rb

@@ -1,7 +0,0 @@
1
-require 'utils'
2
-
3
-class Hash
4
-  def recursively_symbolize_keys
5
-    Utils.recursively_symbolize_keys self
6
-  end
7
-end

+ 46 - 0
lib/serialize_and_normalize.rb

@@ -0,0 +1,46 @@
1
+module SerializeAndNormalize
2
+  extend ActiveSupport::Concern
3
+
4
+  module ClassMethods
5
+    def serialize_and_normalize(*column_names)
6
+      column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name|
7
+        setup_name = "setup_#{column_name}".to_sym
8
+        normalize_name = "normalize_#{column_name}".to_sym
9
+        validate_name = "validate_#{column_name}".to_sym
10
+
11
+        serialize column_name
12
+        after_initialize setup_name
13
+        before_validation normalize_name
14
+        before_save normalize_name
15
+        validate validate_name
16
+
17
+        class_eval <<-RUBY
18
+          def #{setup_name}
19
+            self[:#{column_name}] ||= ActiveSupport::HashWithIndifferentAccess.new
20
+          end
21
+
22
+          def #{validate_name}
23
+            # Implement me in your subclass.
24
+          end
25
+
26
+          def #{normalize_name}
27
+            self.#{column_name} = self[:#{column_name}]
28
+          end
29
+
30
+          def #{column_name}=(data)
31
+            data = (JSON.parse(data) rescue data) if data.is_a?(String)
32
+
33
+            case data
34
+              when ActiveSupport::HashWithIndifferentAccess
35
+                self[:#{column_name}] = data
36
+              when Hash
37
+                self[:#{column_name}] = ActiveSupport::HashWithIndifferentAccess.new(data)
38
+              else
39
+                self[:#{column_name}] = data
40
+            end
41
+          end
42
+        RUBY
43
+      end
44
+    end
45
+  end
46
+end

+ 0 - 43
lib/serialize_and_symbolize.rb

@@ -1,43 +0,0 @@
1
-module SerializeAndSymbolize
2
-  extend ActiveSupport::Concern
3
-
4
-  module ClassMethods
5
-    def serialize_and_symbolize(*column_names)
6
-      column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name|
7
-        setup_name = "setup_#{column_name}".to_sym
8
-        symbolize_name = "symbolize_#{column_name}".to_sym
9
-        validate_name = "validate_#{column_name}".to_sym
10
-
11
-        serialize column_name
12
-        after_initialize setup_name
13
-        before_validation symbolize_name
14
-        before_save symbolize_name
15
-        validate validate_name
16
-
17
-        class_eval <<-RUBY
18
-          def #{setup_name}
19
-            self[:#{column_name}] ||= {}
20
-          end
21
-
22
-          def #{validate_name}
23
-            # Implement me in your subclass.
24
-          end
25
-
26
-          def #{symbolize_name}
27
-            self.#{column_name} = self[:#{column_name}]
28
-          end
29
-
30
-          def #{column_name}=(data)
31
-            if data.is_a?(String)
32
-              self[:#{column_name}] = JSON.parse(data).recursively_symbolize_keys rescue {}
33
-            elsif data.is_a?(Hash)
34
-              self[:#{column_name}] = data.recursively_symbolize_keys
35
-            else
36
-              self[:#{column_name}] = data
37
-            end
38
-          end
39
-        RUBY
40
-      end
41
-    end
42
-  end
43
-end

+ 0 - 11
lib/utils.rb

@@ -21,17 +21,6 @@ module Utils
21 21
     end
22 22
   end
23 23
 
24
-  def self.recursively_symbolize_keys(object)
25
-    case object
26
-      when Hash
27
-        object.inject({}) {|memo, (k, v)| memo[String === k ? k.to_sym : k] = recursively_symbolize_keys(v); memo }
28
-      when Array
29
-        object.map { |item| recursively_symbolize_keys item }
30
-      else
31
-        object
32
-    end
33
-  end
34
-
35 24
   def self.interpolate_jsonpaths(value, data)
36 25
     value.gsub(/<[^>]+>/).each { |jsonpath|
37 26
       Utils.values_at(data, jsonpath[1..-2]).first.to_s

+ 1 - 1
spec/controllers/agents_controller_spec.rb

@@ -23,7 +23,7 @@ describe AgentsController do
23 23
       sign_in users(:bob)
24 24
       post :handle_details_post, :id => agents(:bob_manual_event_agent).to_param, :payload => { :foo => "bar" }
25 25
       JSON.parse(response.body).should == { "success" => true }
26
-      agents(:bob_manual_event_agent).events.last.payload.should == { :foo => "bar" }
26
+      agents(:bob_manual_event_agent).events.last.payload.should == { 'foo' => "bar" }
27 27
     end
28 28
 
29 29
     it "can only be accessed by the Agent's owner" do

+ 3 - 3
spec/controllers/user_location_updates_controller_spec.rb

@@ -8,7 +8,7 @@ describe UserLocationUpdatesController do
8 8
 
9 9
   it "should create events without requiring login" do
10 10
     post :create, :user_id => users(:bob).to_param, :secret => "my_secret", :longitude => 123, :latitude => 45, :something => "else"
11
-    @agent.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" }
11
+    @agent.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" }
12 12
     @agent.events.last.lat.should == 45
13 13
     @agent.events.last.lng.should == 123
14 14
   end
@@ -18,7 +18,7 @@ describe UserLocationUpdatesController do
18 18
     @jane_agent.save!
19 19
 
20 20
     post :create, :user_id => users(:bob).to_param, :secret => "my_secret", :longitude => 123, :latitude => 45, :something => "else"
21
-    @agent.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" }
21
+    @agent.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" }
22 22
     @jane_agent.events.should be_empty
23 23
   end
24 24
 
@@ -33,7 +33,7 @@ describe UserLocationUpdatesController do
33 33
 
34 34
     lambda {
35 35
       post :create, :user_id => users(:bob).to_param, :secret => "my_secret2", :longitude => 123, :latitude => 45, :something => "else"
36
-      @agent2.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" }
36
+      @agent2.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" }
37 37
     }.should_not change { @agent.events.count }
38 38
   end
39 39
 end

+ 2 - 2
spec/controllers/webhooks_controller_spec.rb

@@ -32,12 +32,12 @@ describe WebhooksController do
32 32
 
33 33
   it "should call receive_webhook" do
34 34
     post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5"
35
-    @agent.reload.memory[:webhook_values].should == { :key => "value", :another_key => "5" }
35
+    @agent.reload.memory[:webhook_values].should == { 'key' => "value", 'another_key' => "5" }
36 36
     response.body.should == "success"
37 37
     response.should be_success
38 38
 
39 39
     post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "not_my_secret", :no => "go"
40
-    @agent.reload.memory[:webhook_values].should_not == { :no => "go" }
40
+    @agent.reload.memory[:webhook_values].should_not == { 'no' => "go" }
41 41
     response.body.should == "failure"
42 42
     response.should be_missing
43 43
   end

+ 19 - 19
spec/models/agents/human_task_agent_spec.rb

@@ -288,7 +288,7 @@ describe Agents::HumanTaskAgent do
288 288
       @checker.send :review_hits
289 289
 
290 290
       assignments.all? {|a| a.approved == true }.should be_false
291
-      @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id }
291
+      @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id }
292 292
     end
293 293
 
294 294
     it "shouldn't do anything if an assignment is missing" do
@@ -306,7 +306,7 @@ describe Agents::HumanTaskAgent do
306 306
       @checker.send :review_hits
307 307
 
308 308
       assignments.all? {|a| a.approved == true }.should be_false
309
-      @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id }
309
+      @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id }
310 310
     end
311 311
 
312 312
     it "should create events when all assignments are ready" do
@@ -328,8 +328,8 @@ describe Agents::HumanTaskAgent do
328 328
       hit.should be_disposed
329 329
 
330 330
       @checker.events.last.payload[:answers].should == [
331
-        {:sentiment => "neutral", :feedback => ""},
332
-        {:sentiment => "happy", :feedback => "Take 2"}
331
+        {'sentiment' => "neutral", 'feedback' => ""},
332
+        {'sentiment' => "happy", 'feedback' => "Take 2"}
333 333
       ]
334 334
 
335 335
       @checker.memory[:hits].should == {}
@@ -338,7 +338,7 @@ describe Agents::HumanTaskAgent do
338 338
     describe "taking majority votes" do
339 339
       before do
340 340
         @checker.options[:take_majority] = "true"
341
-        @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
341
+        @checker.memory[:hits] = { "JH3132836336DHG" => @event.id }
342 342
         mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
343 343
       end
344 344
 
@@ -372,14 +372,14 @@ describe Agents::HumanTaskAgent do
372 372
         assignments.all? {|a| a.approved == true }.should be_true
373 373
 
374 374
         @checker.events.last.payload[:answers].should == [
375
-          { :sentiment => "sad", :age_range => "<50" },
376
-          { :sentiment => "neutral", :age_range => ">50" },
377
-          { :sentiment => "happy", :age_range => ">50" },
378
-          { :sentiment => "happy", :age_range => ">50" }
375
+          { 'sentiment' => "sad", 'age_range' => "<50" },
376
+          { 'sentiment' => "neutral", 'age_range' => ">50" },
377
+          { 'sentiment' => "happy", 'age_range' => ">50" },
378
+          { 'sentiment' => "happy", 'age_range' => ">50" }
379 379
         ]
380 380
 
381
-        @checker.events.last.payload[:counts].should == { :sentiment => { :happy => 2, :sad => 1, :neutral => 1 }, :age_range => { :">50" => 3, :"<50" => 1 } }
382
-        @checker.events.last.payload[:majority_answer].should == { :sentiment => "happy", :age_range => ">50" }
381
+        @checker.events.last.payload[:counts].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } }
382
+        @checker.events.last.payload[:majority_answer].should == { 'sentiment' => "happy", 'age_range' => ">50" }
383 383
         @checker.events.last.payload.should_not have_key(:average_answer)
384 384
 
385 385
         @checker.memory[:hits].should == {}
@@ -421,16 +421,16 @@ describe Agents::HumanTaskAgent do
421 421
         assignments.all? {|a| a.approved == true }.should be_true
422 422
 
423 423
         @checker.events.last.payload[:answers].should == [
424
-          { :rating => "1" },
425
-          { :rating => "3" },
426
-          { :rating => "5.1" },
427
-          { :rating => "2" },
428
-          { :rating => "2" }
424
+          { 'rating' => "1" },
425
+          { 'rating' => "3" },
426
+          { 'rating' => "5.1" },
427
+          { 'rating' => "2" },
428
+          { 'rating' => "2" }
429 429
         ]
430 430
 
431
-        @checker.events.last.payload[:counts].should == { :rating => { :"1" => 1, :"2" => 2, :"3" => 1, :"4" => 0, :"5.1" => 1 } }
432
-        @checker.events.last.payload[:majority_answer].should == { :rating => "2" }
433
-        @checker.events.last.payload[:average_answer].should == { :rating => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
431
+        @checker.events.last.payload[:counts].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } }
432
+        @checker.events.last.payload[:majority_answer].should == { 'rating' => "2" }
433
+        @checker.events.last.payload[:average_answer].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
434 434
 
435 435
         @checker.memory[:hits].should == {}
436 436
       end

+ 1 - 1
spec/models/agents/sentiment_agent_spec.rb

@@ -53,7 +53,7 @@ describe Agents::SentimentAgent do
53 53
         it "checks if content key is working fine" do
54 54
             @checker.receive([@event])
55 55
             Event.last.payload[:content].should == "value1"
56
-            Event.last.payload[:original_event].should == {:message => "value1"}
56
+            Event.last.payload[:original_event].should == { 'message' => "value1" }
57 57
         end
58 58
         it "should handle multiple events" do
59 59
             event1 = Event.new

+ 7 - 7
spec/models/agents/twitter_stream_agent_spec.rb

@@ -53,7 +53,7 @@ describe Agents::TwitterStreamAgent do
53 53
         @agent.memory[:filter_counts] = {:keyword1 => 2, :keyword2 => 3, :keyword3 => 4}
54 54
         @agent.save!
55 55
         @agent.process_tweet('keyword1', {:text => "something", :user => {:name => "Mr. Someone"}})
56
-        @agent.reload.memory[:filter_counts].should == {:keyword1 => 3, :keyword2 => 3}
56
+        @agent.reload.memory[:filter_counts].should == { 'keyword1' => 3, 'keyword2' => 3 }
57 57
       end
58 58
     end
59 59
 
@@ -64,9 +64,9 @@ describe Agents::TwitterStreamAgent do
64 64
         }.should change { @agent.events.count }.by(1)
65 65
 
66 66
         @agent.events.last.payload.should == {
67
-          :filter => 'keyword1',
68
-          :text => "something",
69
-          :user => {:name => "Mr. Someone"}
67
+          'filter' => 'keyword1',
68
+          'text' => "something",
69
+          'user' => { 'name' => "Mr. Someone" }
70 70
         }
71 71
       end
72 72
 
@@ -79,9 +79,9 @@ describe Agents::TwitterStreamAgent do
79 79
         }.should change { @agent.events.count }.by(1)
80 80
 
81 81
         @agent.events.last.payload.should == {
82
-          :filter => 'keyword1-1',
83
-          :text => "something",
84
-          :user => {:name => "Mr. Someone"}
82
+          'filter' => 'keyword1-1',
83
+          'text' => "something",
84
+          'user' => { 'name' => "Mr. Someone" }
85 85
         }
86 86
       end
87 87
     end